{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "079206aa-322f-422f-9efe-7a840ff5d4db",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<frozen importlib._bootstrap>:241: RuntimeWarning: scipy._lib.messagestream.MessageStream size changed, may indicate binary incompatibility. Expected 56 from C header, got 64 from PyObject\n",
      "/Users/pachaar/opt/anaconda3/envs/env_twitter/lib/python3.10/site-packages/deeplake/util/check_latest_version.py:32: UserWarning: A newer version of deeplake (3.8.20) is available. It's recommended that you update to the latest version using `pip install -U deeplake`.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import textwrap\n",
    "from dotenv import load_dotenv\n",
    "import re\n",
    "\n",
    "# Load environment variables\n",
    "load_dotenv()\n",
    "\n",
    "# Fetch and set API keys\n",
    "openai_api_key = os.getenv(\"OPENAI_API_KEY\")\n",
    "active_loop_token = os.getenv(\"ACTIVELOOP_TOKEN\")\n",
    "dataset_path = os.getenv(\"DATASET_PATH\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5c3db9fb-1afc-44b3-9538-04cd6881afae",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n"
     ]
    }
   ],
   "source": [
    "from llama_index import download_loader\n",
    "\n",
    "WikipediaReader = download_loader(\"WikipediaReader\")\n",
    "\n",
    "loader = WikipediaReader()\n",
    "\n",
    "documents = loader.load_data(pages=['Delhi', 'Mumbai'])\n",
    "print(len(documents))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "23f20252-46bb-41fd-b9c2-7c3a861268aa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "146\n"
     ]
    }
   ],
   "source": [
    "from llama_index.node_parser import SimpleNodeParser\n",
    "\n",
    "\n",
    "# Initialize the parser\n",
    "parser = SimpleNodeParser.from_defaults(chunk_size=512, chunk_overlap=20)\n",
    "\n",
    "# Parse documents into nodes\n",
    "nodes = parser.get_nodes_from_documents(documents)\n",
    "print(len(nodes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cfd9dc04-d4a9-4d48-8ec0-13acd860fd5d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Your Deep Lake dataset has been successfully created!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "-"
     ]
    }
   ],
   "source": [
    "from llama_index.vector_stores import DeepLakeVectorStore\n",
    "\n",
    "my_activeloop_org_id = \"\"\n",
    "my_activeloop_dataset_name = \"LlamaIndex-101\"\n",
    "dataset_path = f\"hub://{my_activeloop_org_id}/{my_activeloop_dataset_name}\"\n",
    "\n",
    "# Create an index over the documnts\n",
    "vector_store = DeepLakeVectorStore(dataset_path=dataset_path, overwrite=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e761aa7f-0e49-4904-b663-25ea6956bd73",
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.storage.storage_context import StorageContext\n",
    "from llama_index import VectorStoreIndex\n",
    "\n",
    "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n",
    "\n",
    "index = VectorStoreIndex.from_documents(\n",
    "    documents, storage_context=storage_context\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9832ed1c-d83a-48af-9795-be02ce9ed9e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Delhi has been historically significant as it has served as the capital of various empires and kingdoms throughout history. It has been a prominent political, cultural, and commercial center in India for centuries. Delhi's historical significance is rooted in its role as a seat of power, witnessing the rise and fall of different dynasties and playing a crucial part in shaping the country's history.\n"
     ]
    }
   ],
   "source": [
    "query_engine = index.as_query_engine()\n",
    "response = query_engine.query(\"What is historical significance of Delhi?\")\n",
    "print( response.response )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a1643cfc-7bc1-44b8-921b-2c1806488ec5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "env_twitter",
   "language": "python",
   "name": "env_twitter"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
